
#include "../inc/HTMLParser.h"
#include "../inc/KeyWordIndex.h"
#include "../inc/PageHistory.h"
#include "../inc/StopWordList.h"
#include "../inc/URLDownloader.h"
#include "../inc/URLHistory.h"
#include "../inc/URLQueue.h"
#include "../inc/URLValidator.h"
#include "../inc/XMLGenerator.h"

#include"../utils/inc/CommandRunner.h"
#include"../utils/inc/FileInputStream.h"
#include"../utils/inc/FileSystem.h"
#include"../utils/inc/HTMLToken.h"
#include"../utils/inc/HTMLTokenizer.h"
#include"../utils/inc/HTTPInputStream.h"
#include"../utils/inc/StringUtil.h"

using namespace std;

int main(int args, char *info[]) {
	if (args != 4){
		cout << "USAGE: <Site to be Crawler> <XML output File> <Stop Word File>" << endl;
		return 1;
	}
	string startAddr(info[1]);

	URLValidator validator(&startAddr); //initialize stuff
	StopWordList stopwords(info[3]);
	URLQueue queue;
	URLHistory history;
	PageHistory pages;
	URLDownloader downloader;
	HTMLParser parser;
	KeyWordIndex index;
	XMLGenerator gen;
	startAddr=string(info[1]);
	URL * curURL = new URL(&startAddr);
	queue.push(*curURL); //Push first value on the stack
	history.add(*curURL);

	LinkedList<string> *links;
	LinkedList<string> *words;
	string* htmlText;
	Page *curPage;
	string *curDesc;
	LinkedList<URL> *curURLList;


	delete curURL;

	try {
		while (!queue.isEmpty()) {
			links = NULL;
			words = NULL;
			htmlText = NULL;
			curPage = NULL;
			curDesc = NULL;
			curURLList = NULL;
			curURL = NULL;

			try {
				curURL = queue.pop();
				htmlText = downloader.download(curURL); //download current url

				links = parser.getLinks(htmlText);
				curDesc = parser.getDescription(htmlText);
				curURLList = new LinkedList<URL> ();

				//validate each link from downloaded page, if valid add too queue and history

				for (int i = 0; i < links->GetSize(); i++) {
					try{
						URL tempURL = validator.validateURL(links->Get(i),curURL);
						if (history.add(tempURL))
							queue.push(tempURL);
						curURLList->Insert(tempURL);
					}
					catch(InvalidSiteException *e){
						delete e;
					}
				}
				curPage = new Page(*curURL, *curDesc, *curURLList);
				pages.add(*curPage); //Add page into processed page list
				delete curPage; //deletes reference to page
				curPage = pages.getPage(*curURL); //Links curPage copied reference in data struc
				words = parser.getWords(htmlText);
				for (int i = 0; i < words->GetSize(); i++) {
					if (!stopwords.contains(words->Get(i)))
						index.add(*(words->Get(i)), *curPage);
				}

			} catch (NetworkException & e) {
				cout << e.GetMessage() << endl;
				//cout << *curURL << endl;
			} catch (InvalidURLException &e) {
				cout << e.GetMessage() << endl;
				//cout << *curURL << endl;
			} catch (CS240Exception & e) {
				cout << e.GetMessage() << endl;
				//cout << *curURL << endl;
			}

			delete curURL;
			delete curDesc;
			delete curURLList;
			delete htmlText;
			delete links;
			delete words;
		}
		gen.generate(info[2], pages, index); //produce XML file
	} catch (CS240Exception & e) {
		cout << e.GetMessage() << endl;
		delete curURL;
		delete curDesc;
		delete curURLList;
		delete htmlText;
		delete links;
		delete words;
	} catch (...) {
		cout << "Unidentified fault" << endl;
		delete curURL;
		delete curDesc;
		delete curURLList;
		delete htmlText;
		delete links;
		delete words;
	}
	cout << endl << "finish" << endl;
	return 0;
}

